# -*- coding: utf-8 -*-

# Scrapy settings for mySpider project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
#     http://doc.scrapy.org/en/latest/topics/settings.html
#     http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html
#     http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html

BOT_NAME = 'mySpider'

SPIDER_MODULES = ['mySpider.spiders']
NEWSPIDER_MODULE = 'mySpider.spiders'


# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'mySpider (+http://www.yourdomain.com)'

# Obey robots.txt rules
ROBOTSTXT_OBEY = False
UM_COOKIE = {'aptk_uid': '927319a90a0489b8c78ea1a6bf3323061499754538', 'um_lang': 'zh', 'cn_1255795631_dplus': '%7B%22distinct_id%22%3A%20%2215dc49b07d7929-0f8b26d691cd54-6315107d-1fa400-15dc49b07d8215%22%2C%22sp%22%3A%20%7B%22%24_sessionid%22%3A%200%2C%22%24_sessionTime%22%3A%201502414441%2C%22%24dp%22%3A%200%2C%22%24_sessionPVTime%22%3A%201502414441%7D%2C%22initial_view_time%22%3A%20%221502238983%22%2C%22initial_referrer%22%3A%20%22http%3A%2F%2Fmobile.umeng.com%2Fapps%2F641000a9fce85e765d9bed55%2Ferror_types%2Fshow%3Ferror_type_id%3D55deb9d567e58ecf9a000146_2473822712237849596_2.7.1.4%22%2C%22initial_referrer_domain%22%3A%20%22mobile.umeng.com%22%7D', 'cn_1259827933_dplus': '%7B%22distinct_id%22%3A%20%2215dc49b07d7929-0f8b26d691cd54-6315107d-1fa400-15dc49b07d8215%22%2C%22sp%22%3A%20%7B%22%24recent_outside_referrer%22%3A%20%22%24direct%22%2C%22%24_sessionid%22%3A%200%2C%22%24_sessionTime%22%3A%201505440001%2C%22%24dp%22%3A%200%2C%22%24_sessionPVTime%22%3A%201505440001%7D%2C%22initial_view_time%22%3A%20%221505440008%22%2C%22initial_referrer%22%3A%20%22%24direct%22%2C%22initial_referrer_domain%22%3A%20%22%24direct%22%7D', 'cn_1260769985_dplus': '%7B%22distinct_id%22%3A%20%2215dc49b07d7929-0f8b26d691cd54-6315107d-1fa400-15dc49b07d8215%22%2C%22sp%22%3A%20%7B%22%24_sessionid%22%3A%200%2C%22%24_sessionTime%22%3A%201507779887%2C%22%24dp%22%3A%200%2C%22%24_sessionPVTime%22%3A%201507779887%7D%7D', '_uab_collina': '150813942723783011429018', 'cna': 'bO+GEafyk10CATywk2voWppb', 'cn_a61627694930aa9c80cf_dplus': '%7B%22distinct_id%22%3A%20%2215dc49b07d7929-0f8b26d691cd54-6315107d-1fa400-15dc49b07d8215%22%2C%22sp%22%3A%20%7B%22%24recent_outside_referrer%22%3A%20%22%24direct%22%2C%22%24_sessionid%22%3A%200%2C%22%24_sessionTime%22%3A%201509961393%2C%22%24dp%22%3A%200%2C%22%24_sessionPVTime%22%3A%201509961393%7D%2C%22initial_view_time%22%3A%20%221502239623%22%2C%22initial_referrer%22%3A%20%22%24direct%22%2C%22initial_referrer_domain%22%3A%20%22%24direct%22%7D', 'cn_1259864772_dplus': '%7B%22distinct_id%22%3A%20%2215dc49b07d7929-0f8b26d691cd54-6315107d-1fa400-15dc49b07d8215%22%2C%22sp%22%3A%20%7B%22%E6%98%AF%E5%90%A6%E7%99%BB%E5%BD%95%22%3A%20false%2C%22USER%22%3A%20%222458006757%40qq.com%22%2C%22%24_sessionid%22%3A%200%2C%22%24_sessionTime%22%3A%201509961709%2C%22%24dp%22%3A%200%2C%22%24_sessionPVTime%22%3A%201509961709%2C%22%24recent_outside_referrer%22%3A%20%22%24direct%22%7D%2C%22initial_view_time%22%3A%20%221502241727%22%2C%22initial_referrer%22%3A%20%22http%3A%2F%2Fmobile.umeng.com%2Fanalytics%22%2C%22initial_referrer_domain%22%3A%20%22mobile.umeng.com%22%7D', 'umplus_uc_token': '1RVld_q3DngLFgZABW9aU3w_8cde7addb60c43b4a6246c793484a15a', 'umplus_uc_loginid': '2458006757%40qq.com', 'CNZZDATA1258498910': '722587307-1508134819-%7C1510018469', 'cn_1258498910_dplus': '%7B%22distinct_id%22%3A%20%2215dc49b07d7929-0f8b26d691cd54-6315107d-1fa400-15dc49b07d8215%22%2C%22sp%22%3A%20%7B%22%24recent_outside_referrer%22%3A%20%22%24direct%22%2C%22%24_sessionid%22%3A%200%2C%22%24_sessionTime%22%3A%201510018912%2C%22%24dp%22%3A%200%2C%22%24_sessionPVTime%22%3A%201510018912%7D%2C%22initial_view_time%22%3A%20%221502240150%22%2C%22initial_referrer%22%3A%20%22https%3A%2F%2Fwww.baidu.com%2Flink%3Furl%3DA_g21z77vkpSE8u0EE-TPpqBXzEpBg-JxyiVp0IemgG%26wd%3D%26eqid%3Db1977af30003964500000005598a6511%22%2C%22initial_referrer_domain%22%3A%20%22www.baidu.com%22%7D', 'UM_distinctid': '15dc49b07d7929-0f8b26d691cd54-6315107d-1fa400-15dc49b07d8215', 'isg': 'AlRUA-iM8gZloWXB-waYOXaZJZIGBXrHkQ_gZ-41s19k2fUjFrhDJzgbreY7'}

MYSQL_HOST = '192.168.2.80'
MYSQL_PORT = 3306
MYSQL_USER = 'root'
MYSQL_PASSWD = ''
UM_DB = 'umeng_xys'

# Configure maximum concurrent requests performed by Scrapy (default: 16)
#CONCURRENT_REQUESTS = 32

# Configure a delay for requests for the same website (default: 0)
# See http://scrapy.readthedocs.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
DOWNLOAD_DELAY = 3 # 下载延迟
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16

# Disable cookies (enabled by default)
#COOKIES_ENABLED = False

# Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False

# Override the default request headers:
DEFAULT_REQUEST_HEADERS = {
    "User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36",
    "Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
    "Accept-Encoding":"gzip, deflate, br",
    "Accept-Language":"zh-CN,zh;q=0.8",
    "cache-control":"max-age=0"
}

# Enable or disable spider middlewares
# See http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
#    'mySpider.middlewares.MyspiderSpiderMiddleware': 543,
#}

# Enable or disable downloader middlewares
# See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html
#DOWNLOADER_MIDDLEWARES = {
#    'mySpider.middlewares.MyCustomDownloaderMiddleware': 543,
#}

# Enable or disable extensions
# See http://scrapy.readthedocs.org/en/latest/topics/extensions.html
#EXTENSIONS = {
#    'scrapy.extensions.telnet.TelnetConsole': None,
#}

# Configure item pipelines
# See http://scrapy.readthedocs.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
   'mySpider.pipelines.MyspiderPipeline': 300,
}

# Enable and configure the AutoThrottle extension (disabled by default)
# See http://doc.scrapy.org/en/latest/topics/autothrottle.html
#AUTOTHROTTLE_ENABLED = True
# The initial download delay
#AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
#AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
#AUTOTHROTTLE_DEBUG = False

# Enable and configure HTTP caching (disabled by default)
# See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = 'httpcache'
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
